ANDROID: percpu-rwsem: enable percpu_sem destruction in atomic context
Calling percpu_free_rwsem in atomic context results in "scheduling while
atomic" bug being triggered:
BUG: scheduling while atomic: klogd/158/0x00000002
...
__schedule_bug+0x191/0x290
schedule_debug+0x97/0x180
__schedule+0xdc/0xba0
schedule+0xda/0x250
schedule_timeout+0x92/0x2d0
__wait_for_common+0x25b/0x430
wait_for_completion+0x1f/0x30
rcu_barrier+0x440/0x4f0
rcu_sync_dtor+0xaa/0x190
percpu_free_rwsem+0x41/0x80
Introduce percpu_rwsem_destroy function to perform semaphore destruction
in a worker thread.
Bug: 161210518
Signed-off-by: Suren Baghdasaryan <surenb@google.com>
Change-Id: I74ac65c2a9962492cd5002d7a019d2aa13a21a8c
diff --git a/include/linux/percpu-rwsem.h b/include/linux/percpu-rwsem.h
index 5fda40f..bf1668f 100644
--- a/include/linux/percpu-rwsem.h
+++ b/include/linux/percpu-rwsem.h
@@ -13,7 +13,14 @@
struct rcu_sync rss;
unsigned int __percpu *read_count;
struct rcuwait writer;
- wait_queue_head_t waiters;
+ /*
+ * destroy_list_entry is used during object destruction when waiters
+ * can't be used, therefore reusing the same space.
+ */
+ union {
+ wait_queue_head_t waiters;
+ struct list_head destroy_list_entry;
+ };
atomic_t block;
#ifdef CONFIG_DEBUG_LOCK_ALLOC
struct lockdep_map dep_map;
@@ -127,8 +134,12 @@
extern int __percpu_init_rwsem(struct percpu_rw_semaphore *,
const char *, struct lock_class_key *);
+/* Can't be called in atomic context. */
extern void percpu_free_rwsem(struct percpu_rw_semaphore *);
+/* Invokes percpu_free_rwsem and frees the semaphore from a worker thread. */
+extern void percpu_rwsem_async_destroy(struct percpu_rw_semaphore *sem);
+
#define percpu_init_rwsem(sem) \
({ \
static struct lock_class_key rwsem_key; \
diff --git a/kernel/locking/percpu-rwsem.c b/kernel/locking/percpu-rwsem.c
index 70a32a5..a3d37bf 100644
--- a/kernel/locking/percpu-rwsem.c
+++ b/kernel/locking/percpu-rwsem.c
@@ -7,6 +7,7 @@
#include <linux/rcupdate.h>
#include <linux/sched.h>
#include <linux/sched/task.h>
+#include <linux/slab.h>
#include <linux/errno.h>
int __percpu_init_rwsem(struct percpu_rw_semaphore *sem,
@@ -268,3 +269,34 @@
rcu_sync_exit(&sem->rss);
}
EXPORT_SYMBOL_GPL(percpu_up_write);
+
+static LIST_HEAD(destroy_list);
+static DEFINE_SPINLOCK(destroy_list_lock);
+
+static void destroy_list_workfn(struct work_struct *work)
+{
+ struct percpu_rw_semaphore *sem, *sem2;
+ LIST_HEAD(to_destroy);
+
+ spin_lock(&destroy_list_lock);
+ list_splice_init(&destroy_list, &to_destroy);
+ spin_unlock(&destroy_list_lock);
+
+ if (list_empty(&to_destroy))
+ return;
+
+ list_for_each_entry_safe(sem, sem2, &to_destroy, destroy_list_entry) {
+ percpu_free_rwsem(sem);
+ kfree(sem);
+ }
+}
+
+static DECLARE_WORK(destroy_list_work, destroy_list_workfn);
+
+void percpu_rwsem_async_destroy(struct percpu_rw_semaphore *sem)
+{
+ spin_lock(&destroy_list_lock);
+ list_add_tail(&sem->destroy_list_entry, &destroy_list);
+ spin_unlock(&destroy_list_lock);
+ schedule_work(&destroy_list_work);
+}
